#!/usr/bin/env python
# -*- coding: utf-8 -*-
__author__ = 'moxuandeng'

import urllib2
import re


# 处理页面标签
class TagKit:
    # 去除img标签, 7位长空格
    removeImg = re.compile('<img.*?>| {7}|')
    # 删除超链接标签
    removeHref = re.compile('<a.*?>|</a>')
    # 把换行符换为\n
    replaceLine = re.compile('<tr>|<div>|</div>|</p>')
    # 制表符换为\t
    replaceTD = re.compile('<td>')
    # 段落开头换为\t加空两格
    replacePara = re.compile('<p.*?>')
    # 将换行符或双换行符替换为\n
    replaceBR = re.compile('<br><br>|<br>')
    # 将其余标签剔除
    removeExtraTag = re.compile('<.*?>')
    # 空格
    replaceSpace = re.compile('&nbsp;')

    def replace(self, x):
        x = re.sub(self.removeImg, "", x)
        x = re.sub(self.removeHref, "", x)
        x = re.sub(self.replaceLine, "\n", x)
        x = re.sub(self.replaceTD, "\t", x)
        x = re.sub(self.replacePara, "\n    ", x)
        x = re.sub(self.replaceBR, "\n", x)
        x = re.sub(self.replaceBR, "", x)
        x = re.sub(self.removeExtraTag, "", x)
        x = re.sub(self.replaceSpace, "  ", x)
        return x.strip()


class BDTB:
    def __init__(self, url, see_lz, floor_tag):
        self.base_url = url
        self.see_lz = '?see_lz=' + str(see_lz)
        self.tagKit = TagKit()
        self.file = None
        self.floor = 1
        self.defaultTitle = u'百度贴吧'
        self.floorTag = floor_tag

    def get_page(self, page_num):
        try:
            url = self.base_url + self.see_lz + '&pn=' + str(page_num)
            request = urllib2.Request(url)
            response = urllib2.urlopen(request)
            page = response.read()
            return page

        except urllib2.URLError, e:
            if hasattr(e, 'reason'):
                print u'百度贴吧连接失败, 错误原因', e.reason
                return None

    def get_title(self, page):
        pattern = re.compile('<h3 class="core_title_txt.*?>(.*?)</h3>', re.S)
        result = re.search(pattern, page)
        if result:
            return result.group(1).strip()
        else:
            return None

    def get_page_num(self, page):
        pattern = re.compile('<li class="l_reply_num.*?</span>.*?<span.*?>(.*?)</span>', re.S)
        result = re.search(pattern, page)
        if result:
            return result.group(1).strip()
        else:
            return None

    def get_page_contents(self, page):
        pattern = re.compile('<div id="post_content_.*?>(.*?)</div>', re.S)
        items = re.findall(pattern, page)
        contents = []
        for item in items:
            content = "\n" + self.tagKit.replace(item) + "\n"
            contents.append(content)
        return contents

    def setFileTitle(self, title):
        if title is not None:
            self.file = open(title + ".txt", "w+")
        else:
            self.file = open(self.defaultTitle + ".txt", "w++")

    def writeDate(self, contents):
        for item in contents:
            if self.floorTag == 1:
                floor_line_desc = u'\n%d楼%s' % (self.floor, '-' * 100)
                self.file.write(floor_line_desc.encode('utf-8'))
            self.file.write(item)
            self.floor += 1

    def start(self):
        indexPage = self.get_page(1)
        page_num = self.get_page_num(indexPage)
        title = self.get_title(indexPage)
        self.setFileTitle(title)
        if page_num is None:
            print u'URL已失效, 请重试'
            return
        try:
            print "该帖子共有%s页" % page_num
            for i in range(1, int(page_num) + 1):
                print "正在写入第%d页数据" % i
                page = self.get_page(i)
                contents = self.get_page_contents(page)
                self.writeDate(contents)
        except IOError, e:
            print '写入文件异常, 原因%s' % e.message
        finally:
            print '写入任务结束'

    def testIndexPage(self):
        page = self.get_page(1)
        title = self.get_title(page)
        page_num = self.get_page_num(page)
        print 'title:', title, 'page_num', page_num
        self.get_page_contents(page)


if __name__ == "__main__":
    base_url = 'http://tieba.baidu.com/p/492160797'
    bdtb = BDTB(base_url, 1, 1)
    bdtb.start()
